include ../build/Makefile-vars

#
# We build a native version of CPython and a WASM version.
# The native version is ONLY used as part of the cross compilation build process
# and not used for anything else.  TODO: Currently, we can't build native Python using
# only zig on our supported architectures, so we use the native toolchain, which is
# really annoying and makes bootstraping a build a little harder.
#
# See https://www.python.org/downloads/
PYTHON_MAJOR = 3
PYTHON_MINOR = 11
PYTHON_PATCH = 2
PYTHON_BETA =
PYTHON_VERSION = ${PYTHON_MAJOR}.${PYTHON_MINOR}.${PYTHON_PATCH}

URL = https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}${PYTHON_BETA}.tar.xz
TARBALL = ${UPSTREAM}/python-${PYTHON_VERSION}${PYTHON_BETA}.tar.xz

ZLIB_NATIVE = ${PACKAGES}/zlib-native/dist/native
LIBEDIT_NATIVE = ${PACKAGES}/libedit-native/dist/native

POSIX_WASM = $(shell cowasm-package-path @cowasm/posix-wasm)
LIBEDIT_WASM = $(shell cowasm-package-path @cowasm/libedit)
LZMA_WASM = $(shell cowasm-package-path @cowasm/lzma)
NCURSES_WASM = $(shell cowasm-package-path @cowasm/ncurses)
TERMCAP_WASM = $(shell cowasm-package-path @cowasm/termcap)
OPENSSL_WASM = $(shell cowasm-package-path @cowasm/openssl)
SQLITE_WASM = $(shell cowasm-package-path @cowasm/sqlite)
BZIP2_WASM = $(shell cowasm-package-path @cowasm/bzip2)
ZLIB_WASM = $(shell cowasm-package-path @cowasm/zlib)
LIBFFI_WASM = $(shell cowasm-package-path @cowasm/libffi)

PYTHON_WASM = ${BIN}/python-wasm

# Either -Oz or -O3 make sense to me...
#   -O0 = exceeds a limit so can't even load the result with webassembly!
# The cPython default opt is "-DNDEBUG -g -fwrapv -O3 -Wall", which is
# coded in the generated Makefile.  I have noticed some major compiler bugs
# with -O3, so we can revisit this later with newer LLVM/zig releases.
# For example:
#   https://github.com/sagemathinc/cowasm/issues/27
# is a bug with -O3 that goes away with -Oz. The diff in performance
# is less than 5% with -O3 over -Oz.
# NDEBUG I think disable assertions (?).

# Use this instead for better debugging (?).

WASM_OPT=-DNDEBUG -fwrapv -Wall -Oz -g
#WASM_OPT=-DNDEBUG -fwrapv -Wall -Oz

# Default target
all: deps native ${BIN}/python-native wasm ${BIN}/python-wasm

${DIST}/.built: native wasm
	touch ${DIST}/.built

include ../build/Makefile-rules

## NATIVE

${BUILD_NATIVE}/.build:: ${TARBALL}
	cp src/Setup.local-native ${BUILD_NATIVE}/Modules/Setup.local
	# This _scproxy is needed macos *only* for pip; on Linux it breaks the build.
	if [ `uname -s` == "Darwin" ]; then echo "_scproxy _scproxy.c" >> ${BUILD_NATIVE}/Modules/Setup.local; fi

${DIST_NATIVE}/.built: ${BUILD_NATIVE}/.build
	cd ../build && make zig
	cd ../lzma-native && make
	cd ../zlib-native && make
	cd ../termcap-native && make
	cd ../libedit-native && make
	cd ${BUILD_NATIVE} \
		&& 	AR="zig ar" \
			CXX="zig c++ ${ZIG_NATIVE_CFLAGS}" \
			CC="zig cc ${ZIG_NATIVE_CFLAGS}" \
			CFLAGS="-I${ZLIB_NATIVE}/include -I${LIBEDIT_NATIVE}/include" \
			LDFLAGS="-L${ZLIB_NATIVE}/lib -L${LIBEDIT_NATIVE}" \
			./configure \
				--prefix=${DIST_NATIVE} \
				--with-ensurepip \
				--disable-shared \
				--with-readline=editline
	# On **some** macOS systems having either HAVE_MKFIFOAT or HAVE_MKNODAT breaks
	# the build due to I guess bugs with zig and __builtin_available.   We don't need
	# these anywhere, since native cpython is just for bootstrapping our WASM build anyways.
	# NOTE: I didn't hit this with 3.11.0.rc2, but did with the actual release.
	echo "#undef HAVE_MKFIFOAT" >> ${BUILD_NATIVE}/pyconfig.h
	echo "#undef HAVE_MKNODAT" >> ${BUILD_NATIVE}/pyconfig.h
	# We **must** explicitly set RUNSHARED on MacOS since it's wrong there (e.g., it misses
	# zlib).  This is to workaround a security feature of MacOS (see
	# https://developer.apple.com/forums/thread/9233). For simplicity we set this on all hosts.
	cd ${BUILD_NATIVE} \
		&&	make -j8 RUNSHARED="DYLD_LIBRARY_PATH=${BUILD_NATIVE}:${DYLD_LIBRARY_PATH} LD_LIBRARY_PATH=${BUILD_NATIVE}:${LD_LIBRARY_PATH}" \
		&&	make RUNSHARED="DYLD_LIBRARY_PATH=${BUILD_NATIVE}:${DYLD_LIBRARY_PATH} LD_LIBRARY_PATH=${BUILD_NATIVE}:${LD_LIBRARY_PATH}" install
	cp ${SRC}/cowasm_bundler.py ${DIST_NATIVE}/lib/python3.11/site-packages
	touch ${DIST_NATIVE}/.built

${BIN}/python-native: bin/python-native ${DIST_NATIVE}/.built
	rm -f ${BIN}/python-native
	ln -s ${CWD}/bin/python-native ${BIN}/python-native
	touch ${BIN}/python-native

# Use "make run-native" to run native python at the command line.
.PHONY: run-native
run-native: ${DIST_NATIVE}/.built ${BIN}/python-native
	${BIN}/python-native


## WASM

${BUILD_WASM}/.patched: ${BUILD_WASM}/.build
	# Create declarations that make everything extension modules need
	# accessible from outside our "python.wasm as a shared library".
	pnpm dlx dylink ${BUILD_WASM} > ${BUILD_WASM}/Programs/libpython.c
	# Scripts to make it easy to change CPython and rebuild iteratively
	ln -sf ${SRC}/rebuild ${BUILD_WASM}/rebuild
	# Copy the config.site, which answers questions needed for
	# cross compiling, without which ./configure won't work.
	cp src/config.site ${BUILD_WASM}
	# Configure how modules are built
	cp src/Setup.local ${BUILD_WASM}/Modules/Setup.local
	# Custom code to run at startup
	cp src/sitecustomize.py ${BUILD_WASM}/Lib
	# Copy over our small cowasm-specific library with functionality for our packages.
	cp ${SRC}/cowasm_bundler.py ${BUILD_WASM}/Lib
	cp ${SRC}/cowasm_importer.py ${BUILD_WASM}/Lib
	# Make empty sys/wait.h so that python's configure will conclude that we have sys/wait.h; we will
	# explicitly add anything that is really used from there to posix-wasm.h
	mkdir ${BUILD_WASM}/sys
	echo '#include "posix-wasm.h"' >  ${BUILD_WASM}/sys/wait.h
	# Apply our patches:
	cd ${BUILD_WASM} && cat ${SRC}/patches/01-main.patch | patch -p1
	cd ${BUILD_WASM} && cat ${SRC}/patches/02-pydoc.patch | patch -p1
	cd ${BUILD_WASM} && cat ${SRC}/patches/03-wasm-assets.patch | patch -p1
	# Apply this patch when we want to take testing to the next level, and fully enable all
	# tests that involve subprocess support.  Our goal is to support subprocesses fully, but
	# it is NOT to support fork, which is just not possible in general with WebAssembly.
	# However, I believe subprocess support is possible in a fully solid robust way.
	cd ${BUILD_WASM} && cat ${SRC}/patches/04-enable-subprocess-tests.patch | patch -p1
	cd ${BUILD_WASM} && cat ${SRC}/patches/05-st_mode.patch | patch -p1
	# Implement 'import platform; platform.architecture()' for wasm.
	cd ${BUILD_WASM} && cat ${SRC}/patches/06-platform.patch | patch -p1
	# Fork and execv on top of posix-node work, but it takes some restrictions to be robust:
	cd ${BUILD_WASM} && cat ${SRC}/patches/07-subprocess.patch | patch -p1
	cd ${BUILD_WASM} && cat ${SRC}/patches/09-set-inheritable.patch | patch -p1
	# Patch 10-setuptools-c++.patch is used, but only *AFTER* ensurepip happens!
	# clang15 is more strict:
	cd ${BUILD_WASM} && cat ${SRC}/patches/12-timemodule-clang15.patch | patch -p1
	# We can't use the modified WASI socket.h headers, since they annoyingly render the
	# data structure useless (why!? - to save a few bytes?).
	# See packages/python-wasm/src/wasm/posix/socket.zig  (which will move soon)
	cd ${BUILD_WASM} && cat ${SRC}/patches/13-socket-unmodified-headers.patch | patch -p1
	# Warn instead of error on attempt to start a thread.
	# cd ${BUILD_WASM} && cat ${SRC}/patches/14-threading-warning.patch | patch -p1
	touch ${BUILD_WASM}/.patched

DEBUG_MODE = ""
# DEBUG_MODE = "-g"
BLDSHARED = cowasm-cc --experimental-pic -shared ${DEBUG_MODE}

${DIST_WASM}/.built-cpython: node_modules ${BUILD_WASM}/.patched ${DIST_NATIVE}/.built ${BIN}/python-native
	# - Important to set CXX even though it's not in the main build so zig is used for C++ extension modules.
	# - with-pymalloc it actually works and seems faster
	# - We use "zig cc -target wasm32-wasi" instead of "zig cc -target wasm32-wasi-musl" since the broken
	#   Python configure.ac script outputs wasm32-wasi instead of wasm32-wasi-musl as the PLATFORM_TRIPLE.
	#   This might be something to fix and upstream, but for now, just not using a triple works.
	# - We use -I (path to source) in the CC/CXX line in addition to CFLAGS so that we can override some
	#   bad global libraries, e.g., systemwide (zig musl) sys/wait.h with a local version.
	# - In Python source they lower the default recursion limit; however, we don't need that since our
	#   runtime is Chromium, not wasitime and we can handle huge limits (see Include/internal/pycore_ceval.h).
	# - enable-ipv6, since I put a lot of work into this for posix-node!
	# --with-pkg-config=no, this is CRITICAL; otherwise, if pkg-config is installed, then python's build system
	#   will use it and pick up system-wide libraries that are not supported by WASM, which is a huge pain
	#   that leads to subtle bugs.  One example of this is "uuid". (TODO: bug report: should be disabled when cross compiling?)
	# - DPy_DEFAULT_RECURSION_LIMIT=700:
	#       - with -Oz, a limit of 1000 causes test_userdict to fail when testing on
	#         nodejs linux (it works on mac).  test_userlist is even worse.
	#       - with -oZ, test_richcmp.py fails with a limit above 750; a limit of max of 720 works on node on linux.
	#         Changing to -O3 makes no difference.
	# - with-ensurepip: we put that since we need it, but the makefile just runs ensurepip on python-native,
	#   so it doesn't help.  That's fine for our application, since we explicitly run ensurepip later.
	# - explicitly specifying BLDSHARED is needed since otherwise linking shared modules doesn't work.
	#   ALSO, note that wasm-strip for fPIC code must be done at link time and can't be done later, due to
	#   it simply not being implemented yet.  Stripping is automatic with cowasm-cc unless you pass -g.
	cd ${BUILD_WASM} && \
		OPT="${WASM_OPT}" \
		CC="cowasm-cc" \
		CXX="cowasm-c++" \
		AR="zig ar" \
		CFLAGS="${DEBUG_MODE} -I${BUILD_WASM} -I${POSIX_WASM} -DPy_DEFAULT_RECURSION_LIMIT=700" \
		BLDSHARED="${BLDSHARED}" \
		CONFIG_SITE=./config.site \
		READELF=true \
		HOSTRUNNER="${BIN}/cowasm" \
		./configure \
			--config-cache \
			--prefix=${DIST_WASM}  \
			--enable-big-digits=30 \
			--enable-optimizations \
			--enable-ipv6 \
			--disable-shared \
			--with-pkg-config=no \
			--with-readline=editline \
			--with-build-python=${BIN}/python-native \
			--with-pymalloc \
			--with-ensurepip \
			--host=wasm32-unknown-wasi \
			--build=`./config.guess`
	cat src/pyconfig.h >> ${BUILD_WASM}/pyconfig.h
	# NOTE: I have seen "error: unable to build WASI libc CRT file: FileNotFound" when using "make -j8",
	# and had it go away when removing parallel build.
	cd ${BUILD_WASM} && \
		LZMA_WASM="${LZMA_WASM}" LIBEDIT_WASM="${LIBEDIT_WASM}" NCURSES_WASM="${NCURSES_WASM}" TERMCAP_WASM="${TERMCAP_WASM}" OPENSSL_WASM="${OPENSSL_WASM}" SQLITE_WASM="${SQLITE_WASM}" BZIP2_WASM="${BZIP2_WASM}" LIBFFI_WASM="${LIBFFI_WASM}" ZLIB_WASM="${ZLIB_WASM}" make -j8
	mkdir -p ${DIST_WASM} && touch ${DIST_WASM}/.built-cpython

${DIST_WASM}/.install-cpython: node_modules ${DIST_WASM}/.built-cpython
	# CRITICAL!  We have to make changes to build/lib.wasi-wasm32-3.11/_sysconfigdata__wasi_wasm32-wasi.py so it
	# matches the configuration we just used.  TODO: Find a better way to fix this, e.g., by configuration or
	# patching something earlier?   This definitely feels like a hack.
	# This is what based on testing actually works:
	#      'LDSHARED': '' --> 'LDSHARED': '${BLDSHARED}'
	# This is idempotent.
	cd ${BUILD_WASM}/build/lib.wasi-wasm32-3.11/ \
		&& sed -i'.bak' "s/'LDSHARED': ''/'LDSHARED': '${BLDSHARED}'/g" _sysconfigdata__wasi_wasm32-wasi.py

	# Install as usual.  Output is huge so we do not show it all.
	cd ${BUILD_WASM} && \
		LZMA_WASM="${LZMA_WASM}" LIBEDIT_WASM="${LIBEDIT_WASM}" NCURSES_WASM="${NCURSES_WASM}" TERMCAP_WASM="${TERMCAP_WASM}" OPENSSL_WASM="${OPENSSL_WASM}"  SQLITE_WASM="${SQLITE_WASM}" BZIP2_WASM="${BZIP2_WASM}" LIBFFI_WASM="${LIBFFI_WASM}" ZLIB_WASM="${ZLIB_WASM}" make install | tail -n 10

	# For some reason (upstream bug?) test-tomllib isn't copied over, but it's nice to have for testing.
	cp -rv ${BUILD_WASM}/Lib/test/test_tomllib ${DIST_WASM}/lib/python3.11/test/
	# Used for extension modules to efficiently access what Python exports.
	cp ${BUILD_WASM}/Programs/libpython.c ${DIST_WASM}/lib
	# Done!
	touch ${DIST_WASM}/.install-cpython

.PHONY:
wasm-cpython: ${DIST_WASM}/.install-cpython

# These are the minimal data files needed to start cPython, which we found via "DEBUG=wasi:open pw-d"!
MINIMAL_FILES = encodings/__init__.pyc encodings/aliases.pyc encodings/utf_8.pyc
MINIMAL_READLINE = termcap lib-dynload/readline.cpython-311-wasm32-wasi.so  rlcompleter.pyc inspect.pyc ast.pyc contextlib.pyc collections/__init__.pyc keyword.pyc operator.pyc reprlib.pyc functools.pyc types.pyc enum.pyc dis.pyc opcode.pyc collections/abc.pyc importlib/__init__.pyc warnings.pyc linecache.pyc tokenize.pyc re/__init__.pyc re/_compiler.pyc re/_parser.pyc re/_constants.pyc re/_casefix.pyc copyreg.pyc token.pyc

# Below we create two zip files:
#
#  ${DIST_WASM}/lib/dist/python-minimal.zip - (size=10K) -- a tiny file with the minimum needed to start python
#  ${DIST_WASM}/lib/dist/python-stdlib.zip - (size= ~ 5MB) -- the full standard libary
#
# We WANT to run wasm_assets.py under web assembly rather than use the native cpython!
# This is just because we can and because it's a *good test*.  It does not work yet.
# To try this, replace
#    && python-native ./Tools/wasm/wasm_assets.py
# below by
#    && python-wasm ./Tools/wasm/wasm_assets.py

${DIST_WASM}/.built:  node_modules ${DIST_WASM}/.install-cpython ${BIN}/python-native
	# Build wasm asset bundle (the pyc files, etc.).
	cd ${BUILD_WASM} \
		&& mkdir -p usr/local/lib/python3.11/lib-dynload/  \
		&& ${BIN}/python-native ./Tools/wasm/wasm_assets.py \
		&& rm -rf ${DIST_WASM}/lib/dist \
		&& mv usr/local/lib ${DIST_WASM}/lib/dist
	# Add termcap for xterm to our zip asset bundle, so readline actually works:
	cp ${SRC}/termcap ${DIST_WASM}/lib/dist/termcap && cd ${DIST_WASM}/lib/dist/ && zip -u python311.zip termcap
	# **TODO:** It is very silly for these to be in the same zip file, obviously, because that means they have to be downloaded,
	# so we should just put them in the main binary.  We'll change that when things are working.
	mkdir -p ${DIST_WASM}/lib/dist/lib-dynload/
	cp -v ${DIST_WASM}/lib/python3.11/lib-dynload/*.so ${DIST_WASM}/lib/dist/lib-dynload/

	# This would run wasm-opt on the so extension modules, but it is not
	# necessarily "worth it", since it only reduces the size by less than 3%.
	#${BIN}/cowasm-opt ${DIST_WASM}/lib/dist/lib-dynload/

	# Unfortunately, wasm-strip removes the critical custom section that makes the so file useful, since NotImplementedError
	#      https://reviews.llvm.org/D73820
	#find ${DIST_WASM}/lib/dist/lib-dynload -type f -name "*.so" | xargs -n1 ${CWD}/node_modules/.bin/wasm-strip
	# Instead, we use the -s option to wasm-ld above.
	# Also mv zip to a more meaningful name.
	cd ${DIST_WASM}/lib/dist/ \
		&& zip -u python311.zip lib-dynload/* \
		&& mv python311.zip python-stdlib.zip

	# Next make the MINIMAL python0.zip, which is all that's needed to start python (e.g., for a
	# jupyter kernel or to back
	# computations, but not a REPL). We also make a version with the readline so, so you
	# can start a terminal, before more data needs to be loaded.
	cd ${DIST_WASM}/lib/dist/ \
		&& mkdir tmp && cd tmp && unzip ../python-stdlib.zip && cp ../termcap .\
		&& zip ../python-minimal.zip ${MINIMAL_FILES} \
		&& zip ../python-readline.zip ${MINIMAL_FILES} ${MINIMAL_READLINE} \
		&& cd .. && rm -rf tmp

	# Add some code at the end of the _sysconfigdata so that it is sufficiently relocatable so that
	# it's possible to use setuptools and build C extensions.  I don't understand why python doesn't
	# already do this -- maybe there is some subtle edge case that makes it impossible to do
	# in general?  Or maybe people don't relocate python installs then build.  I don't know.
	cat ${SRC}/patches/17-fix-sysconfigdata.py >> ${DIST_WASM}/lib/python3.11/_sysconfigdata__wasi_wasm32-wasi.py

	# We also copy the posix-wasm.h that was used for the build, so that building C extensions isn't
	# complicated by trying to find this.
	cp -v ${POSIX_WASM}/*.h ${DIST_WASM}/include/python3.11/

	touch ${DIST_WASM}/.built


###########
# Installing pip and setuptools
# We can only do this after ../packages/python-wasm is built, which depends on dylink
# (which depends on cpython) and many other things.  So this target is invoked later.
# NOTE: If you do "make clean-wasm", then "make", be sure to also do "make pip".
###########
${DIST_WASM}/lib/python3.11/site-packages/setuptools/.built: ${DIST_WASM}/.built
	rm -rf ${DIST_WASM}/lib/python*/site-packages/*pip* ${DIST_WASM}/lib/python*/site-packages/*setuptools*
	${CWD}/bin/python-wasm -m ensurepip
	# We apply one patch to support building C++ extensions.  Without this, the numpy
	# build fails pretty quickly. This is needed due to our hacky approach to -fPIC.
	cd ${DIST_WASM} && cat ${SRC}/patches/10-setuptools-c++.patch | patch -p1
	# It would be much nicer to implement a better mmap.  But for now, let's just patch it out:
	cd ${DIST_WASM} && cat ${SRC}/patches/15-pip-no-mmap.patch | patch -p1
	# It would be better to have a better fallback for threads that works in some cases so we
	# don't need this. But for now, just remove it:
	cd ${DIST_WASM} && cat ${SRC}/patches/16-pip-no-auto-refresh-progress.patch  | patch -p1
	touch ${DIST_WASM}/lib/python3.11/site-packages/setuptools/.built

pip: ${DIST_WASM}/lib/python3.11/site-packages/setuptools/.built
.PHONY: pip


# Run the full official Python test suite on the wasm build.  We can't just do
# "make test" in build/wasm, since that only support wasmtime and emscripten.
# Also, we can't use our WASM python to orchestrate this (yet), since the
# test suite runner uses threads (and anyways, we like the speedup from multithreading).
# However, you set line 543 to
#         self.ns.use_mp = 0
# of dist/wasm/lib/python3.11/test/libregrtest/main.py then it works, but of course
# with a single process, which is very slow. So we stick with python-native for testing.
RUN_TESTS = _PYTHON_PROJECT_BASE=${BUILD_WASM} \
	_PYTHON_HOST_PLATFORM=wasi-wasm32 \
	PYTHONPATH=${DIST_WASM}/lib/python3.11 \
	_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata__wasi_wasm32-wasi \
	${DIST_NATIVE}/bin/python3 \
	./Tools/scripts/run_tests.py \
	 --python=${PYTHON_WASM}

# We use a restricted PATH for running tests.  On Linux I had a lot of trouble
# with some random binaries in random places causing hangs, and this provides
# a cleaner environment.
PNPM = $(shell which pnpm)
NODE = $(shell which node)
TEST_PATH=${BIN}:/bin:/usr/bin:$(shell dirname ${PNPM}):$(shell dirname ${NODE})

test-all: wasm native
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS}
.PHONY: test-all


# I got this list using this on the output:
#     grep passed out | awk '{print $(NF - 1)}' | tr '\n' ' '
SUPPORTED_TEST_DESC = "the **supported-by-python-wasm** cpython test suites"
SUPPORTED_TESTS = test_atexit test_base64 test_calendar test_cgitb  test_compile test_eof test_fstring test_graphlib test_gzip test_hash test_import test_inspect  test_module test_ntpath test_popen test_posixpath test_py_compile test_pydoc  test_quopri  test_script_helper test_source_encoding test_struct test_tabnanny test_tarfile test_trace test_unicodedata test_weakref test___all__ test___future__ test__locale test__opcode test__osx_support test_abc test_abstract_numbers test_aifc test_argparse test_array test_ast test_audioop test_augassign test_baseexception test_bdb test_bigaddrspace test_bigmem test_binascii test_binop test_bisect test_bool test_buffer test_bufio test_bz2 test_call test_cgi test_charmapcodec test_class test_cmath test_cmd test_code_module test_codeccallbacks test_codecencodings_cn test_codecencodings_hk test_codecencodings_iso2022 test_codecencodings_jp test_codecencodings_kr test_codecencodings_tw test_codecmaps_cn test_codecmaps_hk test_codecmaps_jp test_codecmaps_kr test_codecmaps_tw test_codeop test_collections test_colorsys test_compare test_complex test_configparser test_contains test_context test_contextlib test_copy test_copyreg test_crashers test_crypt test_csv test_dataclasses test_datetime test_dbm test_dbm_dumb test_decimal test_decorators test_defaultdict test_deque test_descr test_descrtut test_dict test_dict_version test_dictcomps test_dictviews test_difflib test_dis test_doctest2 test_dynamic test_dynamicclassattribute test_eintr test_email test_ensurepip test_enum test_enumerate test_errno test_except_star test_exception_group test_exception_hierarchy test_exception_variations test_extcall test_file test_filecmp test_fileinput test_fileutils test_finalization test_float test_flufl test_fnmatch test_format test_fractions test_frame test_frozen test_funcattrs test_functools test_future test_future3 test_future4 test_future5 test_generator_stop test_genericalias test_genericclass test_genericpath test_genexps test_getopt test_getpath test_gettext test_global test_grammar test_hashlib test_heapq test_hmac test_html test_htmlparser test_http_cookiejar test_http_cookies test_imghdr test_index test_int test_int_literal test_ipaddress test_isinstance test_iter test_iterlen test_itertools test_keyword test_keywordonlyarg  test_lib2to3 test_linecache test_list test_listcomps test_lltrace test_locale test_long test_longexp test_lzma test_mailcap test_math test_memoryio test_memoryview test_metaclass test_mimetypes test_minidom test_modulefinder test_multibytecodec test_named_expressions test_netrc  test_numeric_tower test_opcache test_opcodes test_operator test_optparse test_ordered_dict test_osx_env test_patma test_peepholer test_peg_generator test_pep646_syntax test_pickle test_picklebuffer test_pickletools test_pipes test_pkg test_pkgutil test_plistlib test_positional_only_arg test_pow test_pprint test_print test_property test_pulldom test_pyclbr test_pyexpat test_raise test_random test_range test_re test_reprlib test_richcmp test_rlcompleter test_sax test_sched test_scope test_secrets test_set test_setcomps test_shelve test_shlex test_slice test_sndhdr test_sort  test_statistics test_strftime test_string test_string_literals test_stringprep test_strptime test_strtod test_structseq test_subclassinit test_sunau test_sundry test_super test_symtable test_syntax test_textwrap test_threadsignals test_timeit  test_tokenize test_tomllib test_tuple test_type_annotations test_type_cache test_type_comments test_typechecks test_types test_ucn test_unary test_unicode_file test_unicode_file_functions test_unicode_identifiers test_univnewlines test_unpack test_unpack_ex test_unparse test_urlparse test_userdict test_userlist test_userstring test_utf8source test_uu test_uuid test_wave test_weakset test_with test_xdrlib test_xml_dom_minicompat test_xml_etree_c test_xxtestfuzz test_yield_from test_zipimport test_zlib test_zoneinfo  test_pstats test_tempfile test_sys_setprofile


${BIN}/python-wasm: ${DIST_WASM}/.built ${CWD}/bin/python-wasm
	ln -sf ${CWD}/bin/python-wasm ${BIN}/python-wasm

.PHONY: python-wasm
python-wasm: ${BIN}/python-wasm

test-pip: pip
	echo "Testing PIP"
	${CWD}/bin/python-wasm -m pip |grep Usage
	echo "Success"
.PHONY: test-pip


# Some tests can be pretty slow, e.g., test_tarfile takes about a minute or more on a FAST computer,
# and doing 4 in parallel on a slow computer.
test: wasm native ${BIN}/python-wasm test-pip
	echo "Running ${SUPPORTED_TEST_DESC}"
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS}  -j4 --timeout=600 ${SUPPORTED_TESTS}
	echo "Ran ${SUPPORTED_TEST_DESC}"
.PHONY: test

PASSED_WITHOUT_SUBPROCESS_DESC = "These tests all passed without subprocess or socket support enabled, but fail with it."
PASSED_WITHOUT_SUBPROCESS = test_timeout test_stat test_robotparser test_nntplib test_largefile test_logging test_audit test_bytes test_c_locale_coercion test_cmd_line_script test_compileall test_coroutines test_exceptions test_gc test_json test_marshal test_runpy test_site  test_traceback test_unicode test_utf8_mode test_warnings

test-passed-without-subprocess: wasm native
	echo "Running ${PASSED_WITHOUT_SUBPROCESS_DESC}"
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS}  --timeout=180 -j4 ${PASSED_WITHOUT_SUBPROCESS}
	echo "Ran ${PASSED_WITHOUT_SUBPROCESS_DESC}"


# One test fails here due to the fact that the PYTHONHOME is different in webworker mode (since the files are in the zip archive).
test-worker: wasm native
	echo "Running using webworker -- ${SUPPORTED_TEST_DESC}"
	cd ${BUILD_WASM} && PATH=${TEST_PATH} PYTHONHOME=${DIST_WASM} ${RUN_TESTS}  -j4 --timeout=180 ${SUPPORTED_TESTS}
	echo "Ran using webworker -- ${SUPPORTED_TEST_DESC}"

# NOTES:
#  test_pydoc - sometimes randomly hangs, mainly on linux.
# test_sys and test_platform -- I think crashes caused by installing too much and running native subprocesses; eventually we won't run anything native... (these work on most platform but I have an ubuntu docker container where they fail)
#
# test_code test_cppext  fail because of libffi not being done.
#
FAILED_TEST_DESC = "All **unsuppported** and known failing cpython test suites: this is less than 10% of the non-skipped cpython test suite.  We want to fix all of these."
FAILED_TESTS = test_code test_cppext  test_readline test_builtin  test_sys test_platform  test_getpass test_codecs   test_distutils test_fileio test_generators test_glob test_imp test_importlib  test_io    test_openpty test_os test_pathlib   test_posix          test_shutil test_signal  test_sqlite3  test_support  test_sysconfig    test_time  test_tracemalloc test_typing   test_unittest  test_xml_etree test_zipapp test_zipfile


test-failed:  wasm native
	echo "Running ${FAILED_TEST_DESC}"
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS} --timeout=180 -j4 ${FAILED_TESTS}
	echo "Ran ${FAILED_TEST_DESC}"


TEST=test_long
test-one:   wasm native
	echo "Use 'make TEST=test_long test-one' to run a specific test, e.g., test_long in this case."
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS} --timeout=60 -v ${TEST}
.PHONY: test-one

test-one-worker: wasm native
	echo "Use 'make TEST=test_long test-one' to run a specific test, e.g., test_long in this case."
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS} -j1 --timeout=60 -v ${TEST}
.PHONY: test-one


SKIPPED_TEST_DESC = "all **skipped** cpython test suites.  Extend what cpython wasm can do to include more of these."
SKIPPED_TESTS = test_asdl_parser test__xxsubinterpreters test_asyncgen test_asynchat test_asyncio test_asyncore test_check_c_globals test_clinic test_cmd_line test_concurrent_futures test_contextlib_async test_ctypes test_curses test_dbm_gnu test_dbm_ndbm test_devpoll test_doctest test_docxmlrpc test_dtrace test_embed test_epoll test_faulthandler test_fcntl test_file_eintr test_fork1 test_ftplib test_gdb test_grp test_httplib test_httpservers test_idle test_imaplib test_interpreters test_ioctl test_kqueue test_launcher test_mailbox test_mmap test_msilib test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_nis test_ossaudiodev test_pdb test_poll test_poplib test_pty test_pwd test_queue test_regrtest test_repl test_resource test_select test_selectors test_smtpd test_smtplib test_smtpnet test_socket test_socketserver test_spwd test_ssl test_stable_abi_ctypes test_startfile test_subprocess test_sys_settrace test_syslog test_tcl test_telnetlib test_thread test_threadedtempfile test_threading test_threading_local test_tix test_tk test_tools test_ttk_guionly test_ttk_textonly test_turtle test_urllib test_urllib2 test_urllib2_localnet test_urllib2net test_urllib_response test_urllibnet test_venv test_wait3 test_wait4 test_webbrowser test_winconsoleio test_winreg test_winsound test_wsgiref test_xmlrpc test_xmlrpc_net test_xxlimited test_zipfile64 test_zipimport_support  test_cprofile test_profile

test-skipped:  wasm native
	echo "Running ${SKIPPED_TEST_DESC}"
	cd ${BUILD_WASM} && PATH=${TEST_PATH} ${RUN_TESTS}  --timeout=60 ${SKIPPED_TESTS}
	echo "Ran ${SKIPPED_TEST_DESC}"


# This is sort of like the 'assets' thing that is part of cpython, but it's
# much more useful (but less small).
${DIST_WASM}/.publishable: wasm
	# Do not need the tests in the published version -- they're huge
	rm -rf ${DIST_WASM}/lib/python3.11/test

	# Graphics -- Remove idle which we are nowhere near supporting (and is big):
	rm -rf ${DIST_WASM}/lib/python3.11/idlelib
	rm -rf ${DIST_WASM}/lib/python3.11/*tkinter
	rm -rf ${DIST_WASM}/lib/python3.11/turtle*

	# Temporarily make it into a package so that we can bundle it
	touch ${DIST_WASM}/lib/python3.11/__init__.py
	# Bundle it -- using python-native since it's more compressed:
	cd ${DIST_WASM}/lib/ && ${CWD}/bin/python-wasm -m cowasm_bundler python3.11
	# Move it out of the way
	mv ${DIST_WASM}/lib/python3.11 ${DIST_WASM}/lib/python3.11.orig
	# Extract bundle
	cd ${DIST_WASM}/lib/ && tar xf python3.11.tar.xz
	# Remove __init__.pyc
	rm -f ${DIST_WASM}/lib/python3.11/__init__.pyc
	# Also site-packages
	mkdir ${DIST_WASM}/lib/python3.11/site-packages
	# Move compiled so files over
	mv ${DIST_WASM}/lib/python3.11.orig/lib-dynload ${DIST_WASM}/lib/python3.11/
	# Compress libpython3.11 to save 20MB decompressed.  Note that this obviously
	# complicates the python-wasm package which has to explicitly copy this out and decompress it.
	xz ${DIST_WASM}/lib/libpython3.11.a
	# Remove stuff
	rm -rf ${DIST_WASM}/lib/python3.11.tar.xz ${DIST_WASM}/lib/python3.11.orig
	touch ${DIST_WASM}/.publishable

.PHONY: publishable
publishable: ${DIST_WASM}/.publishable
